#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import sys
import argparse
import logging.config

from config import  config
from logger import worklog
from statProfile import stat
from modelProfile import modelProfile

from myinit import spark


def main():
    data = spark.read.parquet(config.Parquet)
    
    #data = spark.sql("""
    #        select rcpt , sender, tag, region, domainrep,authuser,attach, usertag, subject, content
    #        from cac_parquet_event
    #        where concat(ymd,LPAD(h,2,0))>={} and concat(ymd,LPAD(h,2,0))<={}
    #         and tag<>0
    #         and rcpt<>''
    #        """.format(config.BeginTime, config.EndTime))
    data.cache()
    data.createOrReplaceTempView('data')
    
    # 统计画像
    stat()
    
    # 模型画像（文本画像）
    modelProfile()

if __name__ == '__main__':
    logging.config.fileConfig(config.LoggingConfPath)
    logging.basicConfig(level=logging.DEBUG)

    print("Application begin to run... pid: {}".format(os.getpid()))
    worklog.debug("Application begin to run... pid: ")
    
    main()

